from typing import List

from selenium import webdriver
import time
import pymysql

# 需要开多线程么？
from selenium.webdriver.remote.webelement import WebElement

chrome = webdriver.Chrome()
conn = pymysql.connect(host='gsy-cloud', user='root', password='rootQWE', database='python', port=32769)
cursor = conn.cursor()

select_sql = "select pro_pk_id,rate_num,is_visit from tb_teacher where (is_visit = '0' or is_visit = '1') and rate_num > 0;"
update_sql = "update tb_teacher set is_visit = '%s' where pro_pk_id = %s ;"
insert_sql = "insert into tb_rate_comment(pro_pk_id,rate_id,quality,difficulty,thumbs_up_num,thumbs_down_num,course_code,comment,comment_phrase,comment_date) values(\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\");"


def get_page(pk_id):
    while True:
        try:
            url = 'https://www.ratemyprofessors.com/ShowRatings.jsp?tid=replacepkid&showMyProfs=true'
            chrome.get(url.replace('replacepkid', str(pk_id)))
            break
        except Exception as e:
            print(e)
    try:
        chrome.find_element_by_xpath('/html/body/div[5]/div/div/button').click()
    except:
        pass
    load_more()


def load_more():
    while True:
        try:
            chrome.execute_script('arguments[0].click();', chrome.find_element_by_xpath(
                "//button[@class='Buttons__Button-sc-19xdot-1 PaginationButton__StyledPaginationButton-txi1dr-1 eaZArN']"))
            time.sleep(10)
        except Exception as e:
            if not str(e).find('no such element'):
                print(e)
            break


def parse_page(id):
    divs_list = chrome.find_elements_by_xpath(
        "//*[@id=\"ratingsList\"]/li/div/div[@class='Rating__RatingBody-sc-1rhvpxz-0 dGrvXb']")
    count = 1
    tea_comment_list = []
    for div in divs_list:
        quality = parse_item(div, './div[2]/div[1]/div[2]')
        difficulty = parse_item(div, './div[2]/div[2]/div[2]')
        thumbs = parse_items(div, ".//div[@class='RatingFooter__HelpTotal-ciwspm-2 kAVFzA']")
        thumbs_up_num, thumbs_down_num = thumbs_ignore_out_of_index(thumbs)
        course_id = parse_item(div, "./div[3]/div[1]/div[1]/div[1]")
        comment = parse_item(div, ".//div[@class='Comments__StyledComments-dzzyvm-0 gRjWel']").replace("\"", "\\\"")
        comment_phrase = parse_phrase(div, ".//div[@class='RatingTags__StyledTags-sc-1boeqx2-0 eLpnFv']")
        date = parse_item(div, "./div[3]/div[1]/div[2]")
        tuple_obj = (
            str(id), str(count), quality, difficulty, thumbs_up_num, thumbs_down_num, course_id, comment,
            comment_phrase,
            date)
        count += 1
        tea_comment_list.append(tuple_obj)
        # print(tuple_obj)
    return tea_comment_list


def thumbs_ignore_out_of_index(items: List[WebElement]):
    try:
        return items[0].text, items[1].text
    except Exception as e:
        print(e)
        return 0, 0


def parse_phrase(div: WebElement, xpath):
    try:
        item = div.find_element_by_xpath(xpath)
        ss = item.text.split('\n')
        ll = []
        for i in ss:
            ll.append(i)
        return '$$'.join(ll)
    except:
        return ''


def parse_item(div: WebElement, xpath_str):
    try:
        item = div.find_element_by_xpath(xpath_str)
        return item.text
    except Exception as e:
        print(e)
        return ''


def parse_items(div: WebElement, xpath_str):
    try:
        item = div.find_elements_by_xpath(xpath_str)
        return item
    except Exception as e:
        print(e)
        return []


def insert_into_table(tea_comment_list):
    count = 0
    for tea_tuple in tea_comment_list:
        try:
            cursor.execute(insert_sql % tea_tuple)
            conn.commit()
            count += 1
        except Exception as e:
            print("错误sql：" + insert_sql % tea_tuple)
            print(e)
    return count


if __name__ == '__main__':
    cursor.execute(select_sql)
    tea_list = cursor.fetchall()
    for row in tea_list:
        pro_pk_id = row[0]
        rate_num = row[1]
        print("[%s]id:%s正在请求访问,至少需要获取%s条数据" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),pro_pk_id,rate_num))
        get_page(pro_pk_id)
        count = insert_into_table(parse_page(pro_pk_id))
        if count >= int(rate_num):
            print("[%s]id:%s解析完成，共插入数据库%d条数据" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),pro_pk_id,count))
            cursor.execute(update_sql % ('2',pro_pk_id))
            conn.commit()
        else:
            print("[%s]id:%s解析不完全，共插入数据库%d条数据" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), pro_pk_id, count))
            cursor.execute(update_sql % ('1',pro_pk_id))
            conn.commit()



